package com.gt.wc

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object WorldCount_01 {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("XX")
    val sc = new SparkContext(conf)
    sc.setLogLevel("ERROR")
    val rdd1: RDD[String] = sc.textFile("./data")
    val rdd2: RDD[(String, Int)] = rdd1.flatMap(a => a.split(" ")).map(a => (a, 1))
    val rdd3: RDD[(String, Iterable[(String, Int)])] = rdd2.groupBy(data => data._1)
    val rdd4: RDD[(String, Int)] = rdd3.map(a => {
      val tuple: (String, Int) = a._2.reduce((x, y) => (x._1, x._2 + y._2))
      tuple
    })


    rdd4.foreach(println)
    sc.stop()
  }

}
